print(Path().cwd())
os.chdir(L_ROOT)
print(Path().cwd())
filename = ZAPPOS_DF_SIMPLIFIED # "zappos-50k-simplified"
df = pd.read_pickle(f"data/{filename}.pkl")
Because we simply want to collect the features output from the model rather than do classification (or some other decision) I replaced the clasiffier head with a simple identity mapper. The simple Identity nn.Module class makes this simple.
Finally, since we are calculating the features, or embedding over 30k images with the net lets load the computations onto our GPU. We need to remember to do this in evaluation mode so Batch Norm / Dropout layers are disabled. [I forgot to do this initally and lost hours trying to figure out why i wasn't getting consistent results]. Setting param.requires_grad = False saves us memory since we aren't going to fit any weights for now, and protects us in case we forget to do a with torch.no_grad() before inference.
Later when we use the full FastAI API this should all be handled elegantly behind the scenes
sz=IMG_SIZES['small']
device = get_cuda()
batch_size = 128
#dls = get_feats_dataloaders(df,128, IMG_SIZE, get_cuda())
dls = get_feats_dataloaders(df,batch_size,sz,device)
model = get_mnetV2_feature_net(to_cuda=True)
df_f = get_all_feats(dls,model)
df_f.head()
save_featsXsize()
df2 = collate_featsXsize(df)
If we've already calculated everything just load it.
query_image = "Shoes/Sneakers and Athletic Shoes/Nike/7716996.288224.jpg"
df2.loc[df2.path==query_image,['path','classes_md']]
The DataBlock performed a number of processing steps to prepare the images for embedding into the MobileNet_v2 space (1280 vector). Lets confirm that we get the same image and MobileNet_v2 features.
That seemed to work well. I'll just wrap it in a simple function for now, though a FastAI Pipeline might work the best in the long run.
mnetv2=model
query_image2 = '/home/ergonyc/Downloads/491212_01.jpg.jpeg'
query_t = load_and_prep_sneaker(path_images/query_image)
test_feats = get_mnet_feature(mnetv2,query_t)
test_feats.shape
Now I have the "embeddings" of the database in the mobileNet_v2 output space. I can do a logistic regression on these vectors (should be identical to mapping these 1000 vectors to 4 categories (Part 3)) but I can also use an approximate KNN in this space to run the SneakerFinder tool.
next steps:
- make KNN functions.. maybe aproximate KNN e.g. Annoy for speed. Or precalculate .
- PCA / tSNE / UMAP the space with categories to visualize embedding
- make widgets
Lets find the nearest neighbors as a proxy for "similar"
I'll start with a simple "gut" test, and point out that thre realy isn't a ground truth to refer to. Remember that the goal of all this is to find some shoes that someone will like, and we are using "similar" as the aproximation of human preference.
Lets use our previously calculated sneaker-features and inspect that the k- nearest neighbors in our embedding space are feel or look "similar".
Personally, I like Jordans so I chose this as my query_image: 
def get_umap_reducer(latents):
reducer = umap.UMAP(random_state=666)
reducer.fit(latents)
return reducer
df=df2
num_neighs = 5
knns = []
reducers = []
for i,sz in enumerate(IMG_SIZES):
print(ABBR[sz])
print(IMG_SIZES[sz])
features = f"features_{ABBR[sz]}"
print(features)
db_feats = np.vstack(df[features].values)
neighs = NearestNeighbors(n_neighbors=num_neighs) #add plus one in case image exists in database
neighs.fit(db_feats)
knns.append(neighs)
reducer = get_umap_reducer(db_feats)
reducers.append(reducer)
Lets take a quick look at the neighbors according to our list:
neighs = knns[0]
distance, nn_index = neighs.kneighbors(test_feats, return_distance=True)
dist = distance.tolist()[0]
df.columns
paths = df[['path','classes_sm','classes_md','classes_lg']]
neighbors = paths.iloc[nn_index.tolist()[0]].copy()
images = [ PILImage.create(path_images/f) for f in neighbors.path]
#PILImage.create(btn_upload.data[-1])
for im in images:
display(im.to_thumb(IMG_SIZE,IMG_SIZE))
similar_images = []
for i,sz in enumerate(IMG_SIZES):
print(ABBR[sz])
print(IMG_SIZES[sz])
features = f"features_{ABBR[sz]}"
print(features)
query_t = load_and_prep_sneaker(path_images/query_image,IMG_SIZES[sz])
query_f = get_mnet_feature(mnetv2,query_t)
similar_images.append( query_neighs(query_f, knns[i], paths, path_images, show=False) )
im = PILImage.create(path_images/query_image)
display(im.to_thumb(IMG_SIZES[sz]))
plot_sneak_neighs(similar_images)
similar_images2 = []
for i,sz in enumerate(IMG_SIZES):
print(ABBR[sz])
print(IMG_SIZES[sz])
features = f"features_{ABBR[sz]}"
print(features)
query_t = load_and_prep_sneaker(path_images/query_image2,IMG_SIZES[sz])
query_f = get_mnet_feature(mnetv2,query_t)
similar_images2.append( query_neighs(query_f, knns[i], paths, path_images, show=False) )
im = PILImage.create(path_images/query_image2)
display(im.to_thumb(IMG_SIZES[sz]))
plot_sneak_neighs(similar_images2)
df.columns
# first simple PCA
pca = PCA(n_components=2)
for i,sz in enumerate(IMG_SIZES):
print(ABBR[sz])
print(IMG_SIZES[sz])
features = f"features_{ABBR[sz]}"
print(features)
data = df[['Category',features]].copy()
db_feats = np.vstack(data[features].values)
# PCA
pca_result = pca.fit_transform(db_feats)
data['pca-one'] = pca_result[:,0]
data['pca-two'] = pca_result[:,1]
print(f"Explained variation per principal component (sz{sz}): {pca.explained_variance_ratio_}")
smpl_fac=.5
#data=df.reindex(rndperm)
plt.figure(figsize=(16,10))
sns.scatterplot(
x="pca-one",
y="pca-two",
hue="Category",
palette=sns.color_palette("hls", 4),
data=data.sample(frac=smpl_fac),
legend="full",
alpha=0.3
)
plt.savefig(f'PCA 2-D sz{sz}')
plt.show()
# get the UMAP on deck
embedding = reducers[i].transform(db_feats)
data['umap-one'] = embedding[:,0]
data['umap-two'] = embedding[:,1]
plt.figure(figsize=(16,10))
sns.scatterplot(
x="umap-one",
y="umap-two",
hue="Category",
palette=sns.color_palette("hls", 4),
data=data.sample(frac=smpl_fac),
legend="full",
alpha=0.3
)
plt.gca().set_aspect('equal', 'datalim')
plt.title(f'UMAP projection of mobileNetV2 embedded UT-Zappos data (sz{sz})', fontsize=24)
plt.savefig('UMAP 2-D sz{sz}')
plt.show()
fn = df.path.values
type(db_feats)
snk2vec = dict(zip(fn,db_feats))
snk2vec[list(snk2vec.keys())[0]]
embedding = get_umap_embedding(db_feats)
snk2umap = dict(zip(fn,embedding))
# from sklearn.manifold import TSNE
# cov_mat =np.cov(vects.T)
# plt.figure(figsize=(10,10))
# sns.set(font_scale=1.5)
# hm = sns.heatmap(cov_mat,
# cbar=True,
# annot=True,
# square=True,
# fmt='.2f',
# annot_kws={'size': 12},
# cmap='coolwarm')
# plt.title('Covariance matrix showing correlation coefficients', size = 18)
# plt.tight_layout()
# plt.show()
# pca_result = pca.fit_transform(df['feats'].values.tolist())
# df['pca-one'] = pca_result[:,0]
# df['pca-two'] = pca_result[:,1]
# df['pca-three'] = pca_result[:,2]
# print('Explained variation per principal component: {}'.format(pca.explained_variance_ratio_))
# #data=df.sample(frac=1.0)
# #data=df.reindex(rndperm)
# data = df
# #df_subset = df
# time_start = time.time()
# tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)
# tsne_results = tsne.fit_transform(db_feats)
# print('t-SNE done! Time elapsed: {} seconds'.format(time.time()-time_start))
# df['tsne-2d-one'] = tsne_results[:,0]
# df['tsne-2d-two'] = tsne_results[:,1]
# plt.figure(figsize=(16,10))
# sns.scatterplot(
# x="tsne-2d-one", y="tsne-2d-two",
# hue="CategoryDir",
# palette=sns.color_palette("hls", 4),
# data=df,
# legend="full",
# alpha=0.3
#)
from sklearn.metrics import confusion_matrix
from seaborn import heatmap
from sklearn.linear_model import LogisticRegression
#Display Confusion Matrix
X_test = np.vstack(df[df.t_t_v=='test']['features_lg'])
y_test = np.vstack(df[df.t_t_v=='test']['Category']).flatten()
# use validate and train for training (no validation here)
X_train = np.vstack(df[df.train | df.validate]['features_lg'])
y_train = np.vstack(df[df.train | df.validate]['Category']).flatten()
clf_log = LogisticRegression(C = 1, multi_class='ovr', max_iter=2000, solver='lbfgs')
clf_log.fit(X_train, y_train)
log_score = clf_log.score(X_test, y_test)
log_ypred = clf_log.predict(X_test)
log_confusion_matrix = confusion_matrix(y_test, log_ypred)
print(log_confusion_matrix)
disp = heatmap(log_confusion_matrix, annot=True, linewidths=0.5, cmap='Blues')
plt.savefig('log_Matrix.png')
plt.figure(figsize=(16,16))
# Plot non-normalized confusion matrix
titles_options = [("Confusion matrix, without normalization", None),
("Normalized confusion matrix", 'true')]
class_names = df.Category.unique()
from sklearn.metrics import plot_confusion_matrix
for title, normalize in titles_options:
disp = plot_confusion_matrix(clf_log, X_test, y_test,
display_labels=class_names,
cmap=plt.cm.Blues,
normalize=normalize)
disp.ax_.set_title(title)
print(title)
print(disp.confusion_matrix)
plt.savefig('log_Matrix2.png')
from nbdev.export import notebook2script
notebook2script()